# Number of unique articles
length(unique(meta_data$authors_short))
## [1] 43
# Number of effect sizes
dim(meta_data)
## [1] 211 159
# Meta sample size
sum(meta_data$n)
## [1] 121215
summary(meta_data$n)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 25.0 92.0 171.0 574.5 393.5 20796.0
# Average group size human condition
summary(meta_data$focus_n1)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 25.0 61.0 88.0 253.1 194.0 10398.0
# Average group size machine condition
summary(meta_data$focus_n2)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 25.0 62.0 88.0 253.5 194.0 10398.0
plot_by_outlets
discipline_plotdata
## # A tibble: 8 x 4
## discipline n pct lbl
## <fct> <int> <dbl> <chr>
## 1 Information Systems 55 0.261 26.07%
## 2 Management 48 0.227 22.75%
## 3 Psychology 43 0.204 20.38%
## 4 Marketing 34 0.161 16.11%
## 5 Medicine 11 0.0521 5.21%
## 6 General Interest 10 0.0474 4.74%
## 7 Transportation 6 0.0284 2.84%
## 8 Law 4 0.0190 1.90%
plot_by_discipline
disciplineou_plotdata
## # A tibble: 33 x 5
## # Groups: discipline [8]
## discipline outlet n pct lbl
## <fct> <fct> <int> <dbl> <chr>
## 1 Law Journal of Crime and Justice 4 1 100%
## 2 Transportation Accident Analysis & Prevention 6 1 100%
## 3 Medicine Medical Decision Making 9 0.818 82%
## 4 Psychology Journal of Behavioral Decision Making 24 0.558 55.8%
## 5 General Interest Nature Human Behavior 4 0.4 40%
## 6 Marketing Journal of Consumer Research 12 0.353 35.3%
## 7 Management International Journal of Selection and As~ 15 0.312 31.2%
## 8 General Interest Heliyon 3 0.3 30%
## 9 General Interest PLOS ONE 3 0.3 30%
## 10 Psychology Cognition 10 0.233 23.3%
## # ... with 23 more rows
plot_by_disciplineou
pubyear_plotdata
## # A tibble: 15 x 4
## pubyear n pct lbl
## <fct> <int> <dbl> <chr>
## 1 2022 10 0.0474 4.74%
## 2 2021 51 0.242 24.17%
## 3 2020 24 0.114 11.37%
## 4 2019 48 0.227 22.75%
## 5 2018 21 0.0995 9.95%
## 6 2017 9 0.0427 4.27%
## 7 2015 3 0.0142 1.42%
## 8 2013 2 0.00948 0.95%
## 9 2012 12 0.0569 5.69%
## 10 2011 15 0.0711 7.11%
## 11 2009 3 0.0142 1.42%
## 12 2007 6 0.0284 2.84%
## 13 2006 4 0.0190 1.90%
## 14 2005 1 0.00474 0.47%
## 15 2002 2 0.00948 0.95%
plot_by_pubyear
pubtype_plotdata
## # A tibble: 3 x 4
## pubtype n pct lbl
## <fct> <int> <dbl> <chr>
## 1 Journal article 186 0.882 88.2%
## 2 Conference proceedings 14 0.0664 6.6%
## 3 Working Paper 11 0.0521 5.2%
plot_by_pubtype
published_plotdata
## # A tibble: 2 x 4
## published n pct lbl
## <fct> <int> <dbl> <chr>
## 1 yes 186 0.882 88%
## 2 no 25 0.118 12%
plot_by_published
aversion_plotdata
## # A tibble: 2 x 4
## aversion n pct lbl
## <fct> <int> <dbl> <chr>
## 1 1 145 0.687 69%
## 2 0 66 0.313 31%
plot_by_aversion
do.call("grid.arrange", c(plist_1, ncol = 2))
summary(meta_data$n)
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 25.0 92.0 171.0 574.5 393.5 20796.0
hist(meta_data$n, main="Histogram for Sample Sizes", xlab="Sample Sizes")
summary(meta_data$m_age)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 18.75 32.25 36.34 35.17 39.58 50.90 32
hist(meta_data$m_age, main="Histogram for Mean Ages", xlab="Mean Ages")
summary(meta_data$percntg_females)
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## 22.60 45.00 49.00 51.89 57.00 100.00 18
hist(meta_data$percntg_females, main="Histogram for Percentages Females", xlab="Percentages Females")
do.call("grid.arrange", c(plist_2, ncol = 2))
domain_plotdata
## # A tibble: 15 x 4
## domain n pct lbl
## <fct> <int> <dbl> <chr>
## 1 management 64 0.303 30.33%
## 2 medicine 41 0.194 19.43%
## 3 multiple 34 0.161 16.11%
## 4 law 13 0.0616 6.16%
## 5 finance 9 0.0427 4.27%
## 6 shopping 9 0.0427 4.27%
## 7 social 9 0.0427 4.27%
## 8 other 8 0.0379 3.79%
## 9 autonomous driving 7 0.0332 3.32%
## 10 airport security 3 0.0142 1.42%
## 11 art 3 0.0142 1.42%
## 12 baking/cooking 3 0.0142 1.42%
## 13 military 3 0.0142 1.42%
## 14 news 3 0.0142 1.42%
## 15 real estate 2 0.00948 0.95%
plot_by_domain
management_plotdata
## # A tibble: 3 x 4
## management n pct lbl
## <fct> <int> <dbl> <chr>
## 1 no 130 0.616 62%
## 2 yes 75 0.355 36%
## 3 <NA> 6 0.0284 3%
plot_by_management
subjobj_plotdata
## # A tibble: 3 x 4
## subj_vs_obj n pct lbl
## <fct> <int> <dbl> <chr>
## 1 objective 153 0.725 72.5%
## 2 subjective 34 0.161 16.1%
## 3 <NA> 24 0.114 11.4%
plot_by_subjobj
severity_plotdata
## # A tibble: 4 x 4
## severity n pct lbl
## <fct> <int> <dbl> <chr>
## 1 medium 77 0.365 36.5%
## 2 high 61 0.289 28.9%
## 3 low 38 0.180 18.0%
## 4 <NA> 35 0.166 16.6%
plot_by_severity
do.call("grid.arrange", c(plist_3, ncol = 2))
iplot_outlet_plot
iplot_discipline_plot
plist_ipd
## [[1]]
##
## [[2]]
aversion_by_year_plot
plist_imd
## [[1]]
##
## [[2]]
##
## [[3]]
##
## [[4]]
##
## [[5]]
##
## [[6]]
##
## [[7]]
##
## [[8]]
##
## [[9]]
plist_isd
## [[1]]
##
## [[2]]
##
## [[3]]
##
## [[4]]
##
## [[5]]
##
## [[6]]
##
## [[7]]
plist_id
## [[1]]
##
## [[2]]
##
## [[3]]
##
## [[4]]
plist_istd
## [[1]]
##
## [[2]]
##
## [[3]]
##
## [[4]]
##
## [[5]]
Main findings in this section (Model Free Evidence)
print(plist[1])
## $outlet
print(boxplotlist[1])
## $outlet
print(plist[2])
## $discipline
print(boxplotlist[2])
## $discipline
print(meanlist[1])
## $discipline
## # A tibble: 8 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 General Interest -0.179
## 2 Information Systems -0.247
## 3 Law -0.107
## 4 Management -0.106
## 5 Marketing -0.373
## 6 Medicine -0.170
## 7 Psychology -0.398
## 8 Transportation -0.631
print(aovlist[1])
## $discipline
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 7 3.45 0.4934 1.434 0.193
## Residuals 203 69.83 0.3440
TukeyHSD(aov(cohens_d ~ discipline, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ discipline, data = meta_data)
##
## $discipline
## diff lwr upr
## Information Systems-General Interest -0.067821818 -0.6853735 0.5497298
## Law-General Interest 0.071575000 -0.9911771 1.1343271
## Management-General Interest 0.072650000 -0.5517909 0.6970909
## Marketing-General Interest -0.193817647 -0.8400443 0.4524090
## Medicine-General Interest 0.008490909 -0.7764035 0.7933854
## Psychology-General Interest -0.218813953 -0.8494832 0.4118553
## Transportation-General Interest -0.451816667 -1.3794628 0.4758294
## Law-Information Systems 0.139396818 -0.7908809 1.0696745
## Management-Information Systems 0.140471818 -0.2143535 0.4952971
## Marketing-Information Systems -0.125995829 -0.5178930 0.2659014
## Medicine-Information Systems 0.076312727 -0.5170117 0.6696372
## Psychology-Information Systems -0.150992135 -0.5166673 0.2146830
## Transportation-Information Systems -0.383994848 -1.1563302 0.3883405
## Management-Law 0.001075000 -0.9337902 0.9359402
## Marketing-Law -0.265392647 -1.2149480 0.6841627
## Medicine-Law -0.063084091 -1.1119434 0.9857752
## Psychology-Law -0.290388953 -1.2294259 0.6486480
## Transportation-Law -0.523391667 -1.6829493 0.6361660
## Marketing-Management -0.266467647 -0.6691335 0.1361982
## Medicine-Management -0.064159091 -0.6646508 0.5363326
## Psychology-Management -0.291463953 -0.6686571 0.0857292
## Transportation-Management -0.524466667 -1.3023216 0.2533882
## Medicine-Marketing 0.202308556 -0.4208067 0.8254239
## Psychology-Marketing -0.024996306 -0.4372548 0.3872622
## Transportation-Marketing -0.257999020 -1.0534490 0.5374509
## Psychology-Medicine -0.227304863 -0.8342707 0.3796610
## Transportation-Medicine -0.460307576 -1.3720044 0.4513893
## Transportation-Psychology -0.233002713 -1.0158664 0.5498610
## p adj
## Information Systems-General Interest 0.9999760
## Law-General Interest 0.9999992
## Management-General Interest 0.9999644
## Marketing-General Interest 0.9839949
## Medicine-General Interest 1.0000000
## Psychology-General Interest 0.9636558
## Transportation-General Interest 0.8111312
## Law-Information Systems 0.9998032
## Management-Information Systems 0.9274765
## Marketing-Information Systems 0.9761743
## Medicine-Information Systems 0.9999296
## Psychology-Information Systems 0.9106921
## Transportation-Information Systems 0.7943760
## Management-Law 1.0000000
## Marketing-Law 0.9894312
## Medicine-Law 0.9999996
## Psychology-Law 0.9809031
## Transportation-Law 0.8643371
## Marketing-Management 0.4664602
## Medicine-Management 0.9999801
## Psychology-Management 0.2635759
## Transportation-Management 0.4411545
## Medicine-Marketing 0.9748210
## Psychology-Marketing 0.9999996
## Transportation-Marketing 0.9749641
## Psychology-Medicine 0.9454164
## Transportation-Medicine 0.7811834
## Transportation-Psychology 0.9846936
print(plist[3])
## $pubtype
print(boxplotlist[3])
## $pubtype
print(meanlist[2])
## $pubtype
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 Conference proceedings 0.0141
## 2 Journal article -0.279
## 3 Working Paper -0.416
print(aovlist[2])
## $pubtype
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 2 1.38 0.6896 1.995 0.139
## Residuals 208 71.91 0.3457
TukeyHSD(aov(cohens_d ~ pubtype, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ pubtype, data = meta_data)
##
## $pubtype
## diff lwr upr
## Journal article-Conference proceedings -0.2933075 -0.6779597 0.09134477
## Working Paper-Conference proceedings -0.4304351 -0.9896559 0.12878581
## Working Paper-Journal article -0.1371276 -0.5678069 0.29355171
## p adj
## Journal article-Conference proceedings 0.1720876
## Working Paper-Conference proceedings 0.1665353
## Working Paper-Journal article 0.7329892
print(plist[4])
## $published
print(boxplotlist[4])
## $published
print(meanlist[3])
## $published
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.175
## 2 yes -0.279
print(aovlist[3])
## $published
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.24 0.2380 0.681 0.41
## Residuals 209 73.05 0.3495
t.test(cohens_d ~ published, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by published
## t = 1.0904, df = 39.057, p-value = 0.2822
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.08884321 0.29667525
## sample estimates:
## mean in group no mean in group yes
## -0.175320 -0.279236
print(plist[5])
## $within_subject
print(boxplotlist[5])
## $within_subject
print(meanlist[4])
## $within_subject
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.256
## 2 yes -0.328
print(aovlist[4])
## $within_subject
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.14 0.1406 0.402 0.527
## Residuals 209 73.14 0.3500
t.test(cohens_d ~ within_subject, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by within_subject
## t = 0.68673, df = 46.112, p-value = 0.4957
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.1389481 0.2828644
## sample estimates:
## mean in group no mean in group yes
## -0.2560106 -0.3279687
print(plist[6])
## $field
print(boxplotlist[6])
## $field
print(meanlist[5])
## $field
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.253
## 2 yes -0.482
print(aovlist[5])
## $field
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.64 0.6401 1.841 0.176
## Residuals 209 72.64 0.3476
t.test(cohens_d ~ field, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by field
## t = 0.88907, df = 12.601, p-value = 0.3906
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.3293329 0.7874491
## sample estimates:
## mean in group no mean in group yes
## -0.2528111 -0.4818692
print(plist[7])
## $dv_cat
print(boxplotlist[7])
## $dv_cat
print(meanlist[6])
## $dv_cat
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 hypothetical outcome -0.227
## 2 real outcome -0.332
## 3 subjective perception -0.265
print(aovlist[6])
## $dv_cat
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 2 0.15 0.0758 0.216 0.806
## Residuals 208 73.13 0.3516
TukeyHSD(aov(cohens_d ~ dv_cat, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ dv_cat, data = meta_data)
##
## $dv_cat
## diff lwr upr
## real outcome-hypothetical outcome -0.10442500 -0.4824002 0.2735502
## subjective perception-hypothetical outcome -0.03768524 -0.3094730 0.2341025
## subjective perception-real outcome 0.06673976 -0.2403073 0.3737868
## p adj
## real outcome-hypothetical outcome 0.7913416
## subjective perception-hypothetical outcome 0.9426722
## subjective perception-real outcome 0.8650492
print(plist[8])
## $subjective_perception
print(boxplotlist[8])
## $subjective_perception
print(meanlist[7])
## $subjective_perception
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.275
## 2 yes -0.264
print(aovlist[7])
## $subjective_perception
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.01 0.0055 0.016 0.9
## Residuals 209 73.28 0.3506
t.test(cohens_d ~ subjective_perception, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by subjective_perception
## t = -0.11556, df = 86.378, p-value = 0.9083
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.2099490 0.1868804
## sample estimates:
## mean in group no mean in group yes
## -0.2753421 -0.2638078
print(plist[9])
## $dv_scaling
print(boxplotlist[9])
## $dv_scaling
print(meanlist[8])
## $dv_scaling
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 continuous -0.293
## 2 discrete 0.127
print(aovlist[8])
## $dv_scaling
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 2.14 2.1447 6.301 0.0128 *
## Residuals 209 71.14 0.3404
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ dv_scaling, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by dv_scaling
## t = -1.6169, df = 12.577, p-value = 0.1307
## alternative hypothesis: true difference in means between group continuous and group discrete is not equal to 0
## 95 percent confidence interval:
## -0.9814309 0.1428399
## sample estimates:
## mean in group continuous mean in group discrete
## -0.2927571 0.1265385
print(plist[10])
## $dv_measurement_type
print(boxplotlist[10])
## $dv_measurement_type
print(meanlist[9])
## $dv_measurement_type
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 behavioral -0.0456
## 2 self-report -0.314
print(aovlist[9])
## $dv_measurement_type
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 2.20 2.1988 6.465 0.0117 *
## Residuals 209 71.09 0.3401
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ dv_measurement_type, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by dv_measurement_type
## t = 2.0777, df = 44.495, p-value = 0.04354
## alternative hypothesis: true difference in means between group behavioral and group self-report is not equal to 0
## 95 percent confidence interval:
## 0.008135755 0.528755796
## sample estimates:
## mean in group behavioral mean in group self-report
## -0.04555135 -0.31399713
t.test(cohens_d ~ dv_measurement_type, data = meta_data, var.equal = TRUE)
##
## Two Sample t-test
##
## data: cohens_d by dv_measurement_type
## t = 2.5426, df = 209, p-value = 0.01173
## alternative hypothesis: true difference in means between group behavioral and group self-report is not equal to 0
## 95 percent confidence interval:
## 0.06030649 0.47658506
## sample estimates:
## mean in group behavioral mean in group self-report
## -0.04555135 -0.31399713
print(plist[11])
## $compensation
print(boxplotlist[11])
## $compensation
print(meanlist[10])
## $compensation
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.300
## 2 yes -0.254
print(aovlist[10])
## $compensation
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.09 0.0888 0.253 0.615
## Residuals 209 73.20 0.3502
t.test(cohens_d ~ compensation, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by compensation
## t = -0.49117, df = 100.62, p-value = 0.6244
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.2302643 0.1388717
## sample estimates:
## mean in group no mean in group yes
## -0.2998424 -0.2541461
print(plist[12])
## $incentive_compatible
print(boxplotlist[12])
## $incentive_compatible
print(meanlist[11])
## $incentive_compatible
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.289
## 2 yes -0.0200
print(aovlist[11])
## $incentive_compatible
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.13 1.1274 3.266 0.0722 .
## Residuals 209 72.16 0.3452
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ incentive_compatible, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by incentive_compatible
## t = -1.7025, df = 18.518, p-value = 0.1054
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.5993298 0.0621815
## sample estimates:
## mean in group no mean in group yes
## -0.28856237 -0.01998824
print(plist[13])
## $preregistered
print(boxplotlist[13])
## $preregistered
print(meanlist[12])
## $preregistered
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.277
## 2 yes -0.226
print(aovlist[12])
## $preregistered
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.08 0.0836 0.239 0.626
## Residuals 209 73.20 0.3502
t.test(cohens_d ~ preregistered, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by preregistered
## t = -0.46205, df = 55.277, p-value = 0.6459
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.2709366 0.1694007
## sample estimates:
## mean in group no mean in group yes
## -0.276548 -0.225780
print(plist[14])
## $online
print(boxplotlist[14])
## $online
print(meanlist[13])
## $online
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.254
## 2 yes -0.298
## 3 <NA> 0.102
print(aovlist[13])
## $online
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.10 0.0976 0.281 0.597
## Residuals 204 70.87 0.3474
## 5 observations deleted due to missingness
t.test(cohens_d ~ online, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by online
## t = 0.52919, df = 197.77, p-value = 0.5973
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.1186933 0.2057591
## sample estimates:
## mean in group no mean in group yes
## -0.2543279 -0.2978608
print(plist[15])
## $students
print(boxplotlist[15])
## $students
print(meanlist[14])
## $students
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.261
## 2 yes -0.335
## 3 <NA> 0.102
print(aovlist[14])
## $students
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.19 0.1852 0.534 0.466
## Residuals 204 70.78 0.3470
## 5 observations deleted due to missingness
t.test(cohens_d ~ students, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by students
## t = 0.83269, df = 77.193, p-value = 0.4076
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.1035481 0.2524025
## sample estimates:
## mean in group no mean in group yes
## -0.2607085 -0.3351357
print(plist[16])
## $location
print(boxplotlist[16])
## $location
print(meanlist[15])
## $location
## # A tibble: 12 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 America -0.388
## 2 Asia -0.731
## 3 Canada -0.671
## 4 China -0.631
## 5 Germany -0.150
## 6 Italy -0.685
## 7 multiple -0.475
## 8 Netherlands -0.0846
## 9 Non-Us -0.071
## 10 Turkey -0.596
## 11 United States -0.184
## 12 <NA> -0.171
print(aovlist[15])
## $location
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 10 6.08 0.6077 1.637 0.103
## Residuals 131 48.65 0.3714
## 69 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ location, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ location, data = meta_data)
##
## $location
## diff lwr upr p adj
## Asia-America -0.34322273 -1.5090052 0.8225597 0.9966006
## Canada-America -0.28303939 -1.1164810 0.5504022 0.9894731
## China-America -0.24258939 -1.2559190 0.7707402 0.9994125
## Germany-America 0.23762727 -1.2971967 1.7724512 0.9999892
## Italy-America -0.29667273 -1.8314967 1.2381512 0.9999140
## multiple-America -0.08692828 -0.8510537 0.6771971 0.9999994
## Netherlands-America 0.30349394 -0.5939258 1.2009137 0.9898045
## Non-Us-America 0.31712727 -0.5552644 1.1895190 0.9824813
## Turkey-America -0.20753939 -1.5080242 1.0929454 0.9999856
## United States-America 0.20363958 -0.4473170 0.8545961 0.9944528
## Canada-Asia 0.06018333 -1.0925732 1.2129399 1.0000000
## China-Asia 0.10063333 -1.1881877 1.3894543 1.0000000
## Germany-Asia 0.58085000 -1.1482848 2.3099848 0.9903166
## Italy-Asia 0.04655000 -1.6825848 1.7756848 1.0000000
## multiple-Asia 0.25629444 -0.8473859 1.3599748 0.9995506
## Netherlands-Asia 0.64671667 -0.5531104 1.8465437 0.7975694
## Non-Us-Asia 0.66035000 -0.5208740 1.8415740 0.7591473
## Turkey-Asia 0.13568333 -1.3892702 1.6606369 0.9999999
## United States-Asia 0.54686231 -0.4817130 1.5754376 0.8108605
## China-Canada 0.04045000 -0.9578665 1.0387665 1.0000000
## Germany-Canada 0.52066667 -1.0042869 2.0456202 0.9890333
## Italy-Canada -0.01363333 -1.5385869 1.5113202 1.0000000
## multiple-Canada 0.19611111 -0.5479900 0.9402123 0.9986517
## Netherlands-Canada 0.58653333 -0.2938990 1.4669657 0.5211897
## Non-Us-Canada 0.60016667 -0.2547405 1.4550738 0.4401527
## Turkey-Canada 0.07550000 -1.2133210 1.3643210 1.0000000
## United States-Canada 0.48667897 -0.1406514 1.1140093 0.2907075
## Germany-China 0.48021667 -1.1500273 2.1104606 0.9965861
## Italy-China -0.05408333 -1.6843273 1.5761606 1.0000000
## multiple-China 0.15566111 -0.7855607 1.0968829 0.9999798
## Netherlands-China 0.54608333 -0.5062346 1.5984013 0.8328428
## Non-Us-China 0.55971667 -0.4713401 1.5907735 0.7904237
## Turkey-China 0.03505000 -1.3767827 1.4468827 1.0000000
## United States-China 0.44622897 -0.4056837 1.2981417 0.8246160
## Italy-Germany -0.53430000 -2.5309329 1.4623329 0.9984654
## multiple-Germany -0.32455556 -1.8127579 1.1636467 0.9997430
## Netherlands-Germany 0.06586667 -1.4949731 1.6267064 1.0000000
## Non-Us-Germany 0.07950000 -1.4670852 1.6260852 1.0000000
## Turkey-Germany -0.44516667 -2.2678348 1.3775015 0.9993002
## United States-Germany -0.03398769 -1.4673763 1.3994009 1.0000000
## multiple-Italy 0.20974444 -1.2784579 1.6979467 0.9999956
## Netherlands-Italy 0.60016667 -0.9606731 2.1610064 0.9737582
## Non-Us-Italy 0.61380000 -0.9327852 2.1603852 0.9672811
## Turkey-Italy 0.08913333 -1.7335348 1.9118015 1.0000000
## United States-Italy 0.50031231 -0.9330763 1.9337009 0.9870432
## Netherlands-multiple 0.39042222 -0.4246997 1.2055442 0.8921766
## Non-Us-multiple 0.40405556 -0.3834271 1.1915382 0.8424374
## Turkey-multiple -0.12061111 -1.3657305 1.1245083 0.9999999
## United States-multiple 0.29056786 -0.2412272 0.8223630 0.7837615
## Non-Us-Netherlands 0.01363333 -0.9037562 0.9310228 1.0000000
## Turkey-Netherlands -0.51103333 -1.8421219 0.8200553 0.9740430
## United States-Netherlands -0.09985436 -0.8099816 0.6102728 0.9999957
## Turkey-Non-Us -0.52466667 -1.8390114 0.7896780 0.9659491
## United States-Non-Us -0.11348769 -0.7917102 0.5647348 0.9999775
## United States-Turkey 0.41117897 -0.7678796 1.5902375 0.9871285
print(plist[17])
## $US
print(boxplotlist[17])
## $US
print(meanlist[16])
## $US
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.410
## 2 yes -0.214
## 3 <NA> -0.234
print(aovlist[16])
## $US
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.13 1.1337 3.222 0.0751 .
## Residuals 122 42.92 0.3518
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 87 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ US, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ US, data = meta_data)
##
## $US
## diff lwr upr p adj
## yes-no 0.196309 -0.02017535 0.4127933 0.0751119
t.test(cohens_d ~ US, data = meta_data, var.equal = TRUE, na.rm = TRUE)
##
## Two Sample t-test
##
## data: cohens_d by US
## t = -1.7951, df = 122, p-value = 0.07511
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.41279333 0.02017535
## sample estimates:
## mean in group no mean in group yes
## -0.4102708 -0.2139618
print(plist[18])
## $sample_type
print(boxplotlist[18])
## $sample_type
print(meanlist[17])
## $sample_type
## # A tibble: 4 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 mturkers/ prolific participants -0.298
## 2 others -0.200
## 3 students -0.335
## 4 <NA> 0.102
print(aovlist[17])
## $sample_type
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 2 0.56 0.2788 0.804 0.449
## Residuals 203 70.41 0.3469
## 5 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ sample_type, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ sample_type, data = meta_data)
##
## $sample_type
## diff lwr upr
## others-mturkers/ prolific participants 0.09827369 -0.1256503 0.3221977
## students-mturkers/ prolific participants -0.03727493 -0.2922109 0.2176611
## students-others -0.13554862 -0.4134373 0.1423400
## p adj
## others-mturkers/ prolific participants 0.5549304
## students-mturkers/ prolific participants 0.9364443
## students-others 0.4835412
print(plist[19])
## $domain
print(boxplotlist[19])
## $domain
print(meanlist[18])
## $domain
## # A tibble: 15 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 airport security -0.0873
## 2 art -0.314
## 3 autonomous driving -0.769
## 4 baking/cooking -0.372
## 5 finance -0.518
## 6 law -0.173
## 7 management -0.203
## 8 medicine -0.397
## 9 military -1.26
## 10 multiple -0.268
## 11 news 0.0334
## 12 other 0.0882
## 13 real estate -0.685
## 14 shopping 0.102
## 15 social 0.0136
print(aovlist[18])
## $domain
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 14 10.08 0.7202 2.233 0.00792 **
## Residuals 196 63.20 0.3225
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
TukeyHSD(aov(cohens_d ~ domain, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ domain, data = meta_data)
##
## $domain
## diff lwr upr
## art-airport security -0.22666667 -1.81943132 1.36609799
## autonomous driving-airport security -0.68185238 -2.02798421 0.66427944
## baking/cooking-airport security -0.28450000 -1.87726465 1.30826465
## finance-airport security -0.43020000 -1.73068689 0.87028689
## law-airport security -0.08605897 -1.33552637 1.16340842
## management-airport security -0.11568229 -1.26803128 1.03666670
## medicine-airport security -0.30962276 -1.47635453 0.85710900
## military-airport security -1.17733333 -2.77009799 0.41543132
## multiple-airport security -0.18082843 -1.35572062 0.99406375
## news-airport security 0.12070000 -1.47206465 1.71346465
## other-airport security 0.17558333 -1.14506735 1.49623402
## real estate-airport security -0.59746667 -2.37823169 1.18329835
## shopping-airport security 0.18905556 -1.11143134 1.48954245
## social-airport security 0.10094444 -1.19954245 1.40143134
## autonomous driving-art -0.45518571 -1.80131754 0.89094611
## baking/cooking-art -0.05783333 -1.65059799 1.53493132
## finance-art -0.20353333 -1.50402023 1.09695356
## law-art 0.14060769 -1.10885970 1.39007509
## management-art 0.11098437 -1.04136462 1.26333337
## medicine-art -0.08295610 -1.24968786 1.08377567
## military-art -0.95066667 -2.54343132 0.64209799
## multiple-art 0.04583824 -1.12905395 1.22073042
## news-art 0.34736667 -1.24539799 1.94013132
## other-art 0.40225000 -0.91840068 1.72290068
## real estate-art -0.37080000 -2.15156502 1.40996502
## shopping-art 0.41572222 -0.88476467 1.71620912
## social-art 0.32761111 -0.97287578 1.62809801
## baking/cooking-autonomous driving 0.39735238 -0.94877944 1.74348421
## finance-autonomous driving 0.25165238 -0.73142331 1.23472807
## law-autonomous driving 0.59579341 -0.31872308 1.51030990
## management-autonomous driving 0.56617009 -0.21041207 1.34275224
## medicine-autonomous driving 0.37222962 -0.42553893 1.16999816
## military-autonomous driving -0.49548095 -1.84161278 0.85065087
## multiple-autonomous driving 0.50102395 -0.30863233 1.31068023
## news-autonomous driving 0.80255238 -0.54357944 2.14868421
## other-autonomous driving 0.85743571 -0.15216315 1.86703458
## real estate-autonomous driving 0.08438571 -1.47967813 1.64844956
## shopping-autonomous driving 0.87090794 -0.11216775 1.85398362
## social-autonomous driving 0.78279683 -0.20027886 1.76587251
## finance-baking/cooking -0.14570000 -1.44618689 1.15478689
## law-baking/cooking 0.19844103 -1.05102637 1.44790842
## management-baking/cooking 0.16881771 -0.98353128 1.32116670
## medicine-baking/cooking -0.02512276 -1.19185453 1.14160900
## military-baking/cooking -0.89283333 -2.48559799 0.69993132
## multiple-baking/cooking 0.10367157 -1.07122062 1.27856375
## news-baking/cooking 0.40520000 -1.18756465 1.99796465
## other-baking/cooking 0.46008333 -0.86056735 1.78073402
## real estate-baking/cooking -0.31296667 -2.09373169 1.46779835
## shopping-baking/cooking 0.47355556 -0.82693134 1.77404245
## social-baking/cooking 0.38544444 -0.91504245 1.68593134
## law-finance 0.34414103 -0.50175240 1.19003445
## management-finance 0.31451771 -0.37994260 1.00897801
## medicine-finance 0.12057724 -0.59749653 0.83865100
## military-finance -0.74713333 -2.04762023 0.55335356
## multiple-finance 0.24937157 -0.48188664 0.98062978
## news-finance 0.55090000 -0.74958689 1.85138689
## other-finance 0.60578333 -0.34210123 1.55366790
## real estate-finance -0.16726667 -1.69222272 1.35768939
## shopping-finance 0.61925556 -0.30032755 1.53883866
## social-finance 0.53114444 -0.38843866 1.45072755
## management-law -0.02962332 -0.62306895 0.56382232
## medicine-law -0.22356379 -0.84447645 0.39734887
## military-law -1.09127436 -2.34074175 0.15819303
## multiple-law -0.09476946 -0.73088357 0.54134466
## news-law 0.20675897 -1.04270842 1.45622637
## other-law 0.26164231 -0.61493499 1.13821960
## real estate-law -0.51140769 -1.99309375 0.97027837
## shopping-law 0.27511453 -0.57077889 1.12100795
## social-law 0.18700342 -0.65889001 1.03289684
## medicine-management -0.19394047 -0.58416088 0.19627993
## military-management -1.06165104 -2.21400003 0.09069795
## multiple-management -0.06514614 -0.47912754 0.34883526
## news-management 0.23638229 -0.91596670 1.38873128
## other-management 0.29126563 -0.44025825 1.02278950
## real estate-management -0.48178437 -1.88254596 0.91897721
## shopping-management 0.30473785 -0.38972246 0.99919815
## social-management 0.21662674 -0.47783357 0.91108704
## military-medicine -0.86771057 -2.03444233 0.29902120
## multiple-medicine 0.12879433 -0.32368264 0.58127131
## news-medicine 0.43032276 -0.73640900 1.59705453
## other-medicine 0.48520610 -0.26877135 1.23918355
## real estate-medicine -0.28784390 -1.70046127 1.12477346
## shopping-medicine 0.49867832 -0.21939544 1.21675208
## social-medicine 0.41056721 -0.30750655 1.12864097
## multiple-military 0.99650490 -0.17838728 2.17139709
## news-military 1.29803333 -0.29473132 2.89079799
## other-military 1.35291667 0.03226598 2.67356735
## real estate-military 0.57986667 -1.20089835 2.36063169
## shopping-military 1.36638889 0.06590199 2.66687578
## social-military 1.27827778 -0.02220912 2.57876467
## news-multiple 0.30152843 -0.87336375 1.47642062
## other-multiple 0.35641176 -0.41013284 1.12295637
## real estate-multiple -0.41663824 -1.83600304 1.00272657
## shopping-multiple 0.36988399 -0.36137422 1.10114219
## social-multiple 0.28177288 -0.44948533 1.01303108
## other-news 0.05488333 -1.26576735 1.37553402
## real estate-news -0.71816667 -2.49893169 1.06259835
## shopping-news 0.06835556 -1.23213134 1.36884245
## social-news -0.01975556 -1.32024245 1.28073134
## real estate-other -0.77305000 -2.31523775 0.76913775
## shopping-other 0.01347222 -0.93441234 0.96135679
## social-other -0.07463889 -1.02252345 0.87324568
## shopping-real estate 0.78652222 -0.73843383 2.31147828
## social-real estate 0.69841111 -0.82654495 2.22336717
## social-shopping -0.08811111 -1.00769421 0.83147199
## p adj
## art-airport security 0.9999999
## autonomous driving-airport security 0.9178298
## baking/cooking-airport security 0.9999990
## finance-airport security 0.9983314
## law-airport security 1.0000000
## management-airport security 1.0000000
## medicine-airport security 0.9998612
## military-airport security 0.4178332
## multiple-airport security 0.9999999
## news-airport security 1.0000000
## other-airport security 1.0000000
## real estate-airport security 0.9980597
## shopping-airport security 0.9999999
## social-airport security 1.0000000
## autonomous driving-art 0.9978922
## baking/cooking-art 1.0000000
## finance-art 0.9999998
## law-art 1.0000000
## management-art 1.0000000
## medicine-art 1.0000000
## military-art 0.7653385
## multiple-art 1.0000000
## news-art 0.9999872
## other-art 0.9993237
## real estate-art 0.9999929
## shopping-art 0.9988473
## social-art 0.9999253
## baking/cooking-autonomous driving 0.9995247
## finance-autonomous driving 0.9999095
## law-autonomous driving 0.6371411
## management-autonomous driving 0.4422395
## medicine-autonomous driving 0.9562787
## military-autonomous driving 0.9949489
## multiple-autonomous driving 0.7160996
## news-autonomous driving 0.7667898
## other-autonomous driving 0.1977467
## real estate-autonomous driving 1.0000000
## shopping-autonomous driving 0.1466386
## social-autonomous driving 0.2918953
## finance-baking/cooking 1.0000000
## law-baking/cooking 0.9999998
## management-baking/cooking 0.9999999
## medicine-baking/cooking 1.0000000
## military-baking/cooking 0.8373303
## multiple-baking/cooking 1.0000000
## news-baking/cooking 0.9999160
## other-baking/cooking 0.9971190
## real estate-baking/cooking 0.9999992
## shopping-baking/cooking 0.9954671
## social-baking/cooking 0.9995023
## law-finance 0.9867181
## management-finance 0.9657740
## medicine-finance 0.9999996
## military-finance 0.8112385
## multiple-finance 0.9976954
## news-finance 0.9808083
## other-finance 0.6677125
## real estate-finance 1.0000000
## shopping-finance 0.5819427
## social-finance 0.8051843
## management-law 1.0000000
## medicine-law 0.9959563
## military-law 0.1628444
## multiple-law 0.9999999
## news-law 0.9999996
## other-law 0.9994611
## real estate-law 0.9973850
## shopping-law 0.9986096
## social-law 0.9999849
## medicine-management 0.9285569
## military-management 0.1074304
## multiple-management 0.9999998
## news-management 0.9999941
## other-management 0.9891444
## real estate-management 0.9974791
## shopping-management 0.9738332
## social-management 0.9991188
## military-medicine 0.4070665
## multiple-medicine 0.9996860
## news-medicine 0.9948449
## other-medicine 0.6568358
## real estate-medicine 0.9999946
## shopping-medicine 0.5284745
## social-medicine 0.8165274
## multiple-military 0.1994825
## news-military 0.2560024
## other-military 0.0386828
## real estate-military 0.9985914
## shopping-military 0.0290837
## social-military 0.0595101
## news-multiple 0.9999067
## other-multiple 0.9574999
## real estate-multiple 0.9995538
## shopping-multiple 0.9186630
## social-multiple 0.9920665
## other-news 1.0000000
## real estate-news 0.9877576
## shopping-news 1.0000000
## social-news 1.0000000
## real estate-other 0.9238779
## shopping-other 1.0000000
## social-other 1.0000000
## shopping-real estate 0.9064423
## social-real estate 0.9624429
## social-shopping 1.0000000
print(plist[20])
## $management
print(boxplotlist[20])
## $management
print(meanlist[19])
## $management
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.319
## 2 yes -0.179
## 3 <NA> -0.229
print(aovlist[19])
## $management
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.93 0.9311 2.637 0.106
## Residuals 203 71.68 0.3531
## 6 observations deleted due to missingness
t.test(cohens_d ~ management, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by management
## t = -1.4721, df = 114.53, p-value = 0.1437
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.3282030 0.0483646
## sample estimates:
## mean in group no mean in group yes
## -0.3192138 -0.1792947
print(plist[21])
## $human_type
print(boxplotlist[21])
## $human_type
print(meanlist[20])
## $human_type
## # A tibble: 7 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 a lay person/ lay people -0.257
## 2 expert(s) -0.350
## 3 human-controlled machine -0.268
## 4 multiple 0.0402
## 5 other -0.150
## 6 other participant(s) 0.169
## 7 participants themselves 0.733
print(aovlist[20])
## $human_type
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 6 8.83 1.4720 4.659 0.000183 ***
## Residuals 204 64.45 0.3159
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
TukeyHSD(aov(cohens_d ~ human_type, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ human_type, data = meta_data)
##
## $human_type
## diff lwr
## expert(s)-a lay person/ lay people -0.09299288 -0.51986888
## human-controlled machine-a lay person/ lay people -0.01089412 -0.86248502
## multiple-a lay person/ lay people 0.29700588 -0.55458502
## other-a lay person/ lay people 0.10630588 -1.14500862
## other participant(s)-a lay person/ lay people 0.42558824 -0.14855424
## participants themselves-a lay person/ lay people 0.98970588 0.05948876
## human-controlled machine-expert(s) 0.08209876 -0.67802663
## multiple-expert(s) 0.38999876 -0.37012663
## other-expert(s) 0.19929876 -0.99165529
## other participant(s)-expert(s) 0.51858111 0.09170511
## participants themselves-expert(s) 1.08269876 0.23541638
## multiple-human-controlled machine 0.30790000 -0.75076641
## other-human-controlled machine 0.11720000 -1.28328402
## other participant(s)-human-controlled machine 0.43648235 -0.41510855
## participants themselves-human-controlled machine 1.00060000 -0.12228529
## other-multiple -0.19070000 -1.59118402
## other participant(s)-multiple 0.12858235 -0.72300855
## participants themselves-multiple 0.69270000 -0.43018529
## other participant(s)-other 0.31928235 -0.93203215
## participants themselves-other 0.88340000 -0.56623868
## participants themselves-other participant(s) 0.56411765 -0.36609947
## upr p adj
## expert(s)-a lay person/ lay people 0.3338831 0.9950206
## human-controlled machine-a lay person/ lay people 0.8406968 1.0000000
## multiple-a lay person/ lay people 1.1485968 0.9444070
## other-a lay person/ lay people 1.3576204 0.9999779
## other participant(s)-a lay person/ lay people 0.9997307 0.2960924
## participants themselves-a lay person/ lay people 1.9199230 0.0288586
## human-controlled machine-expert(s) 0.8422242 0.9999091
## multiple-expert(s) 1.1501242 0.7277213
## other-expert(s) 1.3902528 0.9988564
## other participant(s)-expert(s) 0.9454571 0.0067778
## participants themselves-expert(s) 1.9299811 0.0034893
## multiple-human-controlled machine 1.3665664 0.9771064
## other-human-controlled machine 1.5176840 0.9999798
## other participant(s)-human-controlled machine 1.2880733 0.7286717
## participants themselves-human-controlled machine 2.1234853 0.1157729
## other-multiple 1.2097840 0.9996494
## other participant(s)-multiple 0.9801733 0.9993641
## participants themselves-multiple 1.8155853 0.5246372
## other participant(s)-other 1.5705969 0.9883629
## participants themselves-other 2.3330387 0.5396541
## participants themselves-other participant(s) 1.4943348 0.5455840
print(plist[22])
## $expert
print(boxplotlist[22])
## $expert
print(meanlist[21])
## $expert
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.103
## 2 yes -0.346
## 3 <NA> 0.348
print(aovlist[21])
## $expert
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.99 1.9914 6.237 0.0133 *
## Residuals 200 63.86 0.3193
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 9 observations deleted due to missingness
t.test(cohens_d ~ expert, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by expert
## t = 2.8976, df = 84.418, p-value = 0.004788
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## 0.07610675 0.40901488
## sample estimates:
## mean in group no mean in group yes
## -0.1033977 -0.3459585
t.test(cohens_d ~ expert, data = meta_data, var.equal = TRUE)
##
## Two Sample t-test
##
## data: cohens_d by expert
## t = 2.4973, df = 200, p-value = 0.01332
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## 0.05103529 0.43408635
## sample estimates:
## mean in group no mean in group yes
## -0.1033977 -0.3459585
print(plist[23])
## $algorithm_dummy
print(boxplotlist[23])
## $algorithm_dummy
print(meanlist[22])
## $algorithm_dummy
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.339
## 2 yes -0.191
print(aovlist[22])
## $algorithm_dummy
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.16 1.1556 3.349 0.0687 .
## Residuals 209 72.13 0.3451
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ algorithm_dummy, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by algorithm_dummy
## t = -1.8311, df = 208.92, p-value = 0.06851
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.30744914 0.01134198
## sample estimates:
## mean in group no mean in group yes
## -0.3391963 -0.1911427
print(plist[24])
## $ai_dummy
print(boxplotlist[24])
## $ai_dummy
print(meanlist[23])
## $ai_dummy
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.321
## 2 yes -0.101
print(aovlist[23])
## $ai_dummy
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.89 1.8923 5.54 0.0195 *
## Residuals 209 71.39 0.3416
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
t.test(cohens_d ~ ai_dummy, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by ai_dummy
## t = -2.4515, df = 93.233, p-value = 0.01609
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.39775444 -0.04174887
## sample estimates:
## mean in group no mean in group yes
## -0.3210805 -0.1013288
t.test(cohens_d ~ ai_dummy, data = meta_data, var.equal = TRUE)
##
## Two Sample t-test
##
## data: cohens_d by ai_dummy
## t = -2.3536, df = 209, p-value = 0.01952
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.40381343 -0.03568988
## sample estimates:
## mean in group no mean in group yes
## -0.3210805 -0.1013288
print(plist[25])
## $textual
print(boxplotlist[25])
## $textual
print(meanlist[24])
## $textual
## # A tibble: 2 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 no -0.182
## 2 yes -0.283
print(aovlist[24])
## $textual
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 0.29 0.2912 0.834 0.362
## Residuals 209 72.99 0.3493
t.test(cohens_d ~ textual, data = meta_data, var.equal = FALSE)
##
## Welch Two Sample t-test
##
## data: cohens_d by textual
## t = 0.81645, df = 42.59, p-value = 0.4188
## alternative hypothesis: true difference in means between group no and group yes is not equal to 0
## 95 percent confidence interval:
## -0.1486173 0.3507121
## sample estimates:
## mean in group no mean in group yes
## -0.1821588 -0.2832062
print(plist[26])
## $severity
print(boxplotlist[26])
## $severity
print(meanlist[25])
## $severity
## # A tibble: 4 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 low -0.0295
## 2 medium -0.278
## 3 high -0.478
## 4 <NA> -0.131
print(aovlist[25])
## $severity
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 2 4.75 2.3727 6.543 0.00182 **
## Residuals 173 62.74 0.3626
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 35 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ severity, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ severity, data = meta_data)
##
## $severity
## diff lwr upr p adj
## medium-low -0.2489130 -0.5311459 0.03331997 0.0959302
## high-low -0.4486311 -0.7428408 -0.15442154 0.0011826
## high-medium -0.1997182 -0.4437381 0.04430174 0.1320253
print(plist[27])
## $subj_vs_obj
print(boxplotlist[27])
## $subj_vs_obj
print(meanlist[26])
## $subj_vs_obj
## # A tibble: 3 x 2
## `modsdata[, i]` name
## <fct> <dbl>
## 1 objective -0.299
## 2 subjective -0.0841
## 3 <NA> -0.324
print(aovlist[26])
## $subj_vs_obj
## Df Sum Sq Mean Sq F value Pr(>F)
## modsdata[, i] 1 1.28 1.2791 3.61 0.059 .
## Residuals 185 65.54 0.3543
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 24 observations deleted due to missingness
TukeyHSD(aov(cohens_d ~ subj_vs_obj, data = meta_data))
## Tukey multiple comparisons of means
## 95% family-wise confidence level
##
## Fit: aov(formula = cohens_d ~ subj_vs_obj, data = meta_data)
##
## $subj_vs_obj
## diff lwr upr p adj
## subjective-objective 0.2144294 -0.00820868 0.4370675 0.0589714
General Notes:
General Notes:
Interpretations:
funnel(as.numeric(meta_data$cohens_d), vi=as.numeric(meta_data$var_d), yaxis="sei", data = meta_data,
main = "Funnel Plot Raw Data", label = 5, offset=1, back = "lightgrey", pch = 20)
General Notes:
General Notes:
Interpretations:
# Estimates Multi-level random-effects meta-regression model, without moderators
multilevel <- rma.mv(cohens_d, var_d, random = ~ 1 |
study_id/es_id, data = meta_data, method="ML")
multilevel
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0782 0.2797 99 no study_id
## sigma^2.2 0.2523 0.5023 211 no study_id/es_id
##
## Test for Heterogeneity:
## Q(df = 210) = 20928.2790, p-val < .0001
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## -0.2464 0.0483 -5.1016 <.0001 -0.3411 -0.1517 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
# intra-class correlation
(0.0782)/(0.0782+0.2523)
## [1] 0.2366112
plot(standresid1, type="o", pch=19, xlab="Observed effect size", ylab="Standardized residual")
plot(cookdist1, type="o", pch=19, xlab="Observed effect size", ylab="Cook's Distance")
## Forest Plot ####
meta_data$cohens_d_plot <- format(round(meta_data$cohens_d, 2), nsmall = 2)
meta_data$cohens_d_plot <- as.numeric(meta_data$cohens_d_plot)
meta_data$pubyear <- as.numeric(as.character(meta_data$pubyear))
forest(multilevel,
annotate = FALSE,
addfit = TRUE,
addpred = FALSE,
showweights = FALSE,
xlim=c(-10,1), # adjust horizontal plot region limits
ilab=cbind(meta_data$pubyear, meta_data$study_no, meta_data$n, meta_data$cohens_d_plot),
ilab.xpos=c(-5.40, -4.80, -4.30, -3.65),
ilab.pos = 4,
xlab = "Standardized Mean Difference",
order="obs",
header = TRUE,
slab = meta_data$authors_short)
op <- par(cex=1, font=2)
text(x = c(-5.40, -4.80, -4.30, -3.65), y = 213, label = c("Year","No.", "N", "d"), pos = 4)
par(op)
funnel(multilevel, yaxis = "sei", main = "Funnel Plot Multi-level Random-Effects Model without Moderators", label = 5, offset=1, back = "lightgrey", pch = 20)
Interpretations:
## Failsafe N ####
fsn(cohens_d, vi=var_d, data = meta_data, type = "Rosenthal")
##
## Fail-safe N Calculation Using the Rosenthal Approach
##
## Observed Significance Level: <.0001
## Target Significance Level: 0.05
##
## Fail-safe N: 261326
# Threshold
(5*211)+10
## [1] 1065
# Rank test of Begg and Mazumdar (1994)
ranktest(multilevel)
##
## Rank Correlation Test for Funnel Plot Asymmetry
##
## Kendall's tau = -0.1308, p = 0.0047
# Egger Test (Regress effect size on s.e. or precision)
eggermodel_d_1 <- summary(lm(cohens_d ~ n, data = meta_data))
eggermodel_d_1
##
## Call:
## lm(formula = cohens_d ~ n, data = meta_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -2.1672 -0.2979 -0.0172 0.4000 1.9883
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) -0.25901207 0.04213389 -6.147 0.00000000394 ***
## n -0.00001377 0.00001888 -0.729 0.467
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.5914 on 209 degrees of freedom
## Multiple R-squared: 0.002539, Adjusted R-squared: -0.002234
## F-statistic: 0.532 on 1 and 209 DF, p-value: 0.4666
outlet_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0000 0.0000 99 no study_id
## sigma^2.2 0.1773 0.4211 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 178) = 15482.8331, p-val < .0001
##
## Test of Moderators (coefficients 2:33):
## QM(df = 32) = 162.6074, p-val < .0001
##
## Model Results:
##
## estimate
## intrcpt -0.4166
## outletAccident Analysis & Prevention -0.2146
## outletAI & Society 0.3321
## outletBig Data & Society -0.3368
## outletBusiness & Information Systems Engineering 0.6576
## outletCognition -0.4928
## outletComputers in Human Behavior 0.3071
## outletEuropean Journal of Marketing 0.1585
## outletHeliyon 0.7068
## outletHuman Factors 0.1690
## outletInternational Journal of Human-Computer Studies 0.1221
## outletInternational Journal of Selection and Assessment 0.1949
## outletJournal of Accounting Research -0.2116
## outletJournal of Behavioral Decision Making -0.0281
## outletJournal of Consumer Psychology -0.2382
## outletJournal of Consumer Research -0.2269
## outletJournal of Crime and Justice 0.3092
## outletJournal of Experimental Psychology: Applied 0.7356
## outletJournal of Forecasting 0.4886
## outletJournal of Management Accounting Research 0.1059
## outletJournal of Marketing 0.2949
## outletJournal of Marketing Research 0.0522
## outletJournal of Public Policy & Marketing 1.7128
## outletMarketing Science -0.3138
## outletMedical Decision Making 0.3353
## outletNature Human Behavior 0.1616
## outletNeuron -0.1579
## outletOrganizational Behavior and Human Decision Processes 0.8291
## outletPLOS ONE -0.1303
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems 0.4166
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society 0.4624
## outletProceedings of the ACM on Human-Computer Interaction 0.4369
## outletTechnological Forecasting and Social Change 0.8388
## se
## intrcpt 0.1286
## outletAccident Analysis & Prevention 0.2166
## outletAI & Society 0.1907
## outletBig Data & Society 0.1834
## outletBusiness & Information Systems Engineering 0.4480
## outletCognition 0.1873
## outletComputers in Human Behavior 0.1858
## outletEuropean Journal of Marketing 0.2811
## outletHeliyon 0.2806
## outletHuman Factors 0.2358
## outletInternational Journal of Human-Computer Studies 0.2497
## outletInternational Journal of Selection and Assessment 0.1700
## outletJournal of Accounting Research 0.2794
## outletJournal of Behavioral Decision Making 0.1564
## outletJournal of Consumer Psychology 0.3253
## outletJournal of Consumer Research 0.1784
## outletJournal of Crime and Justice 0.2475
## outletJournal of Experimental Psychology: Applied 0.1920
## outletJournal of Forecasting 0.2789
## outletJournal of Management Accounting Research 0.2779
## outletJournal of Marketing 0.2098
## outletJournal of Marketing Research 0.2474
## outletJournal of Public Policy & Marketing 0.3319
## outletMarketing Science 0.2469
## outletMedical Decision Making 0.1932
## outletNature Human Behavior 0.2480
## outletNeuron 0.3585
## outletOrganizational Behavior and Human Decision Processes 0.1863
## outletPLOS ONE 0.2761
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems 0.4432
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society 0.3286
## outletProceedings of the ACM on Human-Computer Interaction 0.1867
## outletTechnological Forecasting and Social Change 0.2794
## zval
## intrcpt -3.2394
## outletAccident Analysis & Prevention -0.9905
## outletAI & Society 1.7415
## outletBig Data & Society -1.8365
## outletBusiness & Information Systems Engineering 1.4678
## outletCognition -2.6311
## outletComputers in Human Behavior 1.6535
## outletEuropean Journal of Marketing 0.5639
## outletHeliyon 2.5187
## outletHuman Factors 0.7166
## outletInternational Journal of Human-Computer Studies 0.4889
## outletInternational Journal of Selection and Assessment 1.1470
## outletJournal of Accounting Research -0.7573
## outletJournal of Behavioral Decision Making -0.1794
## outletJournal of Consumer Psychology -0.7322
## outletJournal of Consumer Research -1.2716
## outletJournal of Crime and Justice 1.2494
## outletJournal of Experimental Psychology: Applied 3.8314
## outletJournal of Forecasting 1.7518
## outletJournal of Management Accounting Research 0.3811
## outletJournal of Marketing 1.4054
## outletJournal of Marketing Research 0.2111
## outletJournal of Public Policy & Marketing 5.1601
## outletMarketing Science -1.2713
## outletMedical Decision Making 1.7351
## outletNature Human Behavior 0.6516
## outletNeuron -0.4406
## outletOrganizational Behavior and Human Decision Processes 4.4497
## outletPLOS ONE -0.4721
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems 0.9398
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society 1.4071
## outletProceedings of the ACM on Human-Computer Interaction 2.3408
## outletTechnological Forecasting and Social Change 3.0021
## pval
## intrcpt 0.0012
## outletAccident Analysis & Prevention 0.3219
## outletAI & Society 0.0816
## outletBig Data & Society 0.0663
## outletBusiness & Information Systems Engineering 0.1422
## outletCognition 0.0085
## outletComputers in Human Behavior 0.0982
## outletEuropean Journal of Marketing 0.5728
## outletHeliyon 0.0118
## outletHuman Factors 0.4736
## outletInternational Journal of Human-Computer Studies 0.6249
## outletInternational Journal of Selection and Assessment 0.2514
## outletJournal of Accounting Research 0.4489
## outletJournal of Behavioral Decision Making 0.8576
## outletJournal of Consumer Psychology 0.4640
## outletJournal of Consumer Research 0.2035
## outletJournal of Crime and Justice 0.2115
## outletJournal of Experimental Psychology: Applied 0.0001
## outletJournal of Forecasting 0.0798
## outletJournal of Management Accounting Research 0.7031
## outletJournal of Marketing 0.1599
## outletJournal of Marketing Research 0.8328
## outletJournal of Public Policy & Marketing <.0001
## outletMarketing Science 0.2036
## outletMedical Decision Making 0.0827
## outletNature Human Behavior 0.5146
## outletNeuron 0.6595
## outletOrganizational Behavior and Human Decision Processes <.0001
## outletPLOS ONE 0.6369
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems 0.3473
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society 0.1594
## outletProceedings of the ACM on Human-Computer Interaction 0.0192
## outletTechnological Forecasting and Social Change 0.0027
## ci.lb
## intrcpt -0.6686
## outletAccident Analysis & Prevention -0.6391
## outletAI & Society -0.0416
## outletBig Data & Society -0.6962
## outletBusiness & Information Systems Engineering -0.2205
## outletCognition -0.8599
## outletComputers in Human Behavior -0.0569
## outletEuropean Journal of Marketing -0.3924
## outletHeliyon 0.1568
## outletHuman Factors -0.2932
## outletInternational Journal of Human-Computer Studies -0.3673
## outletInternational Journal of Selection and Assessment -0.1382
## outletJournal of Accounting Research -0.7593
## outletJournal of Behavioral Decision Making -0.3347
## outletJournal of Consumer Psychology -0.8758
## outletJournal of Consumer Research -0.5766
## outletJournal of Crime and Justice -0.1759
## outletJournal of Experimental Psychology: Applied 0.3593
## outletJournal of Forecasting -0.0581
## outletJournal of Management Accounting Research -0.4387
## outletJournal of Marketing -0.1164
## outletJournal of Marketing Research -0.4327
## outletJournal of Public Policy & Marketing 1.0622
## outletMarketing Science -0.7976
## outletMedical Decision Making -0.0434
## outletNature Human Behavior -0.3245
## outletNeuron -0.8605
## outletOrganizational Behavior and Human Decision Processes 0.4639
## outletPLOS ONE -0.6714
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems -0.4521
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society -0.1817
## outletProceedings of the ACM on Human-Computer Interaction 0.0711
## outletTechnological Forecasting and Social Change 0.2912
## ci.ub
## intrcpt -0.1645
## outletAccident Analysis & Prevention 0.2100
## outletAI & Society 0.7058
## outletBig Data & Society 0.0226
## outletBusiness & Information Systems Engineering 1.5356
## outletCognition -0.1257
## outletComputers in Human Behavior 0.6712
## outletEuropean Journal of Marketing 0.7095
## outletHeliyon 1.2567
## outletHuman Factors 0.6311
## outletInternational Journal of Human-Computer Studies 0.6115
## outletInternational Journal of Selection and Assessment 0.5281
## outletJournal of Accounting Research 0.3361
## outletJournal of Behavioral Decision Making 0.2785
## outletJournal of Consumer Psychology 0.3994
## outletJournal of Consumer Research 0.1228
## outletJournal of Crime and Justice 0.7943
## outletJournal of Experimental Psychology: Applied 1.1119
## outletJournal of Forecasting 1.0352
## outletJournal of Management Accounting Research 0.6505
## outletJournal of Marketing 0.7061
## outletJournal of Marketing Research 0.5372
## outletJournal of Public Policy & Marketing 2.3634
## outletMarketing Science 0.1700
## outletMedical Decision Making 0.7140
## outletNature Human Behavior 0.6478
## outletNeuron 0.5447
## outletOrganizational Behavior and Human Decision Processes 1.1943
## outletPLOS ONE 0.4108
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems 1.2852
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society 1.1065
## outletProceedings of the ACM on Human-Computer Interaction 0.8027
## outletTechnological Forecasting and Social Change 1.3865
##
## intrcpt **
## outletAccident Analysis & Prevention
## outletAI & Society .
## outletBig Data & Society .
## outletBusiness & Information Systems Engineering
## outletCognition **
## outletComputers in Human Behavior .
## outletEuropean Journal of Marketing
## outletHeliyon *
## outletHuman Factors
## outletInternational Journal of Human-Computer Studies
## outletInternational Journal of Selection and Assessment
## outletJournal of Accounting Research
## outletJournal of Behavioral Decision Making
## outletJournal of Consumer Psychology
## outletJournal of Consumer Research
## outletJournal of Crime and Justice
## outletJournal of Experimental Psychology: Applied ***
## outletJournal of Forecasting .
## outletJournal of Management Accounting Research
## outletJournal of Marketing
## outletJournal of Marketing Research
## outletJournal of Public Policy & Marketing ***
## outletMarketing Science
## outletMedical Decision Making .
## outletNature Human Behavior
## outletNeuron
## outletOrganizational Behavior and Human Decision Processes ***
## outletPLOS ONE
## outletProceedings of the 2019 CHI Conference on Human Factors in Computing Systems
## outletProceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society
## outletProceedings of the ACM on Human-Computer Interaction *
## outletTechnological Forecasting and Social Change **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
discipline_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0631 0.2512 99 no study_id
## sigma^2.2 0.2475 0.4975 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 203) = 19428.7658, p-val < .0001
##
## Test of Moderators (coefficients 2:8):
## QM(df = 7) = 10.1948, p-val = 0.1778
##
## Model Results:
##
## estimate se zval pval ci.lb
## intrcpt -0.2229 0.1926 -1.1576 0.2470 -0.6004
## disciplineInformation Systems 0.0614 0.2183 0.2812 0.7785 -0.3665
## disciplineLaw 0.1156 0.4031 0.2869 0.7742 -0.6744
## disciplineManagement 0.1801 0.2185 0.8243 0.4098 -0.2482
## disciplineMarketing -0.1374 0.2189 -0.6279 0.5300 -0.5664
## disciplineMedicine 0.0479 0.2789 0.1716 0.8637 -0.4988
## disciplinePsychology -0.1481 0.2145 -0.6905 0.4899 -0.5685
## disciplineTransportation -0.3880 0.3047 -1.2735 0.2028 -0.9851
## ci.ub
## intrcpt 0.1545
## disciplineInformation Systems 0.4893
## disciplineLaw 0.9056
## disciplineManagement 0.6084
## disciplineMarketing 0.2915
## disciplineMedicine 0.5946
## disciplinePsychology 0.2723
## disciplineTransportation 0.2091
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
pubyear_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0655 0.2559 99 no study_id
## sigma^2.2 0.2539 0.5039 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20738.2794, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.6765, p-val = 0.0306
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -46.3067 21.2990 -2.1741 0.0297 -88.0520 -4.5615 *
## pubyear 0.0228 0.0106 2.1625 0.0306 0.0021 0.0435 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
pubtype_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0698 0.2642 99 no study_id
## sigma^2.2 0.2531 0.5031 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 208) = 20692.6921, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 3.2342, p-val = 0.1985
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt 0.0444 0.1884 0.2354 0.8139 -0.3249 0.4136
## pubtypeJournal article -0.3011 0.1950 -1.5443 0.1225 -0.6833 0.0811
## pubtypeWorking Paper -0.4769 0.2798 -1.7046 0.0883 -1.0253 0.0714 .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
published_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0767 0.2769 99 no study_id
## sigma^2.2 0.2529 0.5029 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20927.4858, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.3118, p-val = 0.5766
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.1718 0.1422 -1.2080 0.2271 -0.4505 0.1070
## publishedyes -0.0844 0.1511 -0.5584 0.5766 -0.3806 0.2118
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
within_subject_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0769 0.2773 99 no study_id
## sigma^2.2 0.2528 0.5028 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20928.2761, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.2925, p-val = 0.5886
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2359 0.0520 -4.5406 <.0001 -0.3378 -0.1341 ***
## within_subjectyes -0.0748 0.1384 -0.5409 0.5886 -0.3461 0.1964
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
# random_model
field_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0729 0.2699 99 no study_id
## sigma^2.2 0.2537 0.5037 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20315.2533, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 1.4417, p-val = 0.2299
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2325 0.0492 -4.7288 <.0001 -0.3289 -0.1361 ***
## fieldyes -0.2440 0.2032 -1.2007 0.2299 -0.6422 0.1543
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_cat_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0806 0.2840 99 no study_id
## sigma^2.2 0.2501 0.5001 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 208) = 20279.7643, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 0.4593, p-val = 0.7948
##
## Model Results:
##
## estimate se zval pval ci.lb
## intrcpt -0.2700 0.1083 -2.4924 0.0127 -0.4823
## dv_catreal outcome -0.0437 0.1673 -0.2609 0.7941 -0.3716
## dv_catsubjective perception 0.0419 0.1181 0.3548 0.7227 -0.1896
## ci.ub
## intrcpt -0.0577 *
## dv_catreal outcome 0.2843
## dv_catsubjective perception 0.2734
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
subjective_perception_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0816 0.2857 99 no study_id
## sigma^2.2 0.2495 0.4995 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20322.2147, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.4231, p-val = 0.5154
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2892 0.0821 -3.5228 0.0004 -0.4501 -0.1283
## subjective_perceptionyes 0.0617 0.0949 0.6505 0.5154 -0.1242 0.2477
##
## intrcpt ***
## subjective_perceptionyes
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_scaling_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0767 0.2769 99 no study_id
## sigma^2.2 0.2431 0.4930 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20689.3842, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 6.9632, p-val = 0.0083
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2801 0.0493 -5.6830 <.0001 -0.3767 -0.1835 ***
## dv_scalingdiscrete 0.4606 0.1746 2.6388 0.0083 0.1185 0.8027 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
dv_mt_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0746 0.2732 99 no study_id
## sigma^2.2 0.2427 0.4927 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20759.4199, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 7.9620, p-val = 0.0048
##
## Model Results:
##
## estimate se zval pval ci.lb
## intrcpt 0.0145 0.1038 0.1393 0.8892 -0.1891
## dv_measurement_typeself-report -0.3231 0.1145 -2.8217 0.0048 -0.5476
## ci.ub
## intrcpt 0.2180
## dv_measurement_typeself-report -0.0987 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
compensation_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0770 0.2775 99 no study_id
## sigma^2.2 0.2521 0.5021 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20927.6131, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.6704, p-val = 0.4129
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.3206 0.1026 -3.1259 0.0018 -0.5217 -0.1196 **
## compensationyes 0.0951 0.1162 0.8188 0.4129 -0.1326 0.3228
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
incentive_compatible_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0719 0.2682 99 no study_id
## sigma^2.2 0.2521 0.5021 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20881.3933, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 2.8262, p-val = 0.0927
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2735 0.0501 -5.4607 <.0001 -0.3717 -0.1753
## incentive_compatibleyes 0.2573 0.1530 1.6811 0.0927 -0.0427 0.5573
##
## intrcpt ***
## incentive_compatibleyes .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
preregistered_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0792 0.2815 99 no study_id
## sigma^2.2 0.2516 0.5016 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20928.2224, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.0247, p-val = 0.8751
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2421 0.0554 -4.3739 <.0001 -0.3506 -0.1336 ***
## preregisteredyes -0.0179 0.1140 -0.1572 0.8751 -0.2413 0.2054
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
m_age_model
##
## Multivariate Meta-Analysis Model (k = 179; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.1156 0.3401 83 no study_id
## sigma^2.2 0.2367 0.4865 179 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 177) = 16691.9554, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.9053, p-val = 0.3414
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.5064 0.2867 -1.7660 0.0774 -1.0684 0.0556 .
## m_age 0.0078 0.0082 0.9515 0.3414 -0.0082 0.0238
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
percntg_females_model
##
## Multivariate Meta-Analysis Model (k = 193; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.1191 0.3451 89 no study_id
## sigma^2.2 0.2179 0.4668 193 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 191) = 19982.4838, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.1909, p-val = 0.6621
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.3409 0.2305 -1.4792 0.1391 -0.7927 0.1108
## percntg_females 0.0019 0.0043 0.4370 0.6621 -0.0065 0.0102
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
online_model
##
## Multivariate Meta-Analysis Model (k = 206; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0747 0.2734 95 no study_id
## sigma^2.2 0.2518 0.5018 206 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 204) = 20455.1362, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.0662, p-val = 0.7970
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2470 0.0708 -3.4870 0.0005 -0.3858 -0.1082 ***
## onlineyes -0.0250 0.0974 -0.2572 0.7970 -0.2159 0.1658
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
students_model
##
## Multivariate Meta-Analysis Model (k = 206; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0722 0.2687 95 no study_id
## sigma^2.2 0.2531 0.5030 206 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 204) = 20507.4422, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.3251, p-val = 0.5685
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2455 0.0549 -4.4727 <.0001 -0.3531 -0.1379 ***
## studentsyes -0.0661 0.1158 -0.5702 0.5685 -0.2931 0.1610
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
location_model
##
## Multivariate Meta-Analysis Model (k = 142; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0827 0.2875 50 no study_id
## sigma^2.2 0.2329 0.4826 142 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 131) = 13034.9118, p-val < .0001
##
## Test of Moderators (coefficients 2:11):
## QM(df = 10) = 23.4366, p-val = 0.0092
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.3904 0.2027 -1.9262 0.0541 -0.7876 0.0069 .
## locationAsia -0.3402 0.4267 -0.7975 0.4252 -1.1765 0.4960
## locationCanada -0.2775 0.2990 -0.9279 0.3535 -0.8635 0.3086
## locationChina -0.2156 0.3138 -0.6872 0.4920 -0.8306 0.3994
## locationGermany 0.2399 0.4957 0.4841 0.6283 -0.7316 1.2115
## locationItaly -0.4084 0.4660 -0.8764 0.3808 -1.3217 0.5049
## locationmultiple -0.3001 0.2614 -1.1480 0.2510 -0.8125 0.2123
## locationNetherlands 0.3059 0.3870 0.7904 0.4293 -0.4526 1.0643
## locationNon-Us 0.6035 0.3143 1.9198 0.0549 -0.0126 1.2196 .
## locationTurkey -0.2044 0.4545 -0.4497 0.6530 -1.0952 0.6865
## locationUnited States 0.3182 0.2218 1.4344 0.1515 -0.1166 0.7529
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
US_model
##
## Multivariate Meta-Analysis Model (k = 124; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0595 0.2440 41 no study_id
## sigma^2.2 0.2499 0.4999 124 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 122) = 11613.9151, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.3688, p-val = 0.0366
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.4185 0.1015 -4.1242 <.0001 -0.6174 -0.2196 ***
## USyes 0.2544 0.1217 2.0902 0.0366 0.0158 0.4930 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
sample_type_model
##
## Multivariate Meta-Analysis Model (k = 206; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0716 0.2676 95 no study_id
## sigma^2.2 0.2527 0.5027 206 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 203) = 20455.1057, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 0.8274, p-val = 0.6612
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2720 0.0664 -4.0990 <.0001 -0.4020 -0.1419 ***
## sample_typeothers 0.0831 0.1176 0.7072 0.4795 -0.1473 0.3136
## sample_typestudents -0.0397 0.1215 -0.3264 0.7441 -0.2778 0.1985
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
domain_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0666 0.2581 99 no study_id
## sigma^2.2 0.2070 0.4550 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 196) = 16979.2065, p-val < .0001
##
## Test of Moderators (coefficients 2:15):
## QM(df = 14) = 41.6825, p-val = 0.0001
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.0474 0.3339 -0.1419 0.8872 -0.7019 0.6071
## domainart -0.2689 0.4518 -0.5953 0.5516 -1.1544 0.6165
## domainautonomous driving -0.7079 0.3922 -1.8051 0.0711 -1.4765 0.0607
## domainbaking/cooking -0.2926 0.4697 -0.6230 0.5333 -1.2131 0.6279
## domainfinance -0.4281 0.3854 -1.1107 0.2667 -1.1834 0.3273
## domainlaw -0.1957 0.3713 -0.5270 0.5982 -0.9233 0.5320
## domainmanagement -0.0030 0.3468 -0.0087 0.9930 -0.6827 0.6767
## domainmedicine -0.3805 0.3463 -1.0987 0.2719 -1.0592 0.2983
## domainmilitary -1.2198 0.4534 -2.6902 0.0071 -2.1086 -0.3311
## domainmultiple -0.1897 0.3525 -0.5382 0.5904 -0.8806 0.5012
## domainnews -0.0455 0.4557 -0.0999 0.9204 -0.9387 0.8476
## domainother 0.1107 0.3894 0.2843 0.7762 -0.6526 0.8740
## domainreal estate -0.6039 0.5207 -1.1598 0.2461 -1.6244 0.4166
## domainshopping 0.3287 0.3898 0.8433 0.3990 -0.4352 1.0926
## domainsocial 0.0589 0.3809 0.1546 0.8772 -0.6876 0.8054
##
## intrcpt
## domainart
## domainautonomous driving .
## domainbaking/cooking
## domainfinance
## domainlaw
## domainmanagement
## domainmedicine
## domainmilitary **
## domainmultiple
## domainnews
## domainother
## domainreal estate
## domainshopping
## domainsocial
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
management_model
##
## Multivariate Meta-Analysis Model (k = 205; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0954 0.3089 95 no study_id
## sigma^2.2 0.2330 0.4827 205 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 203) = 20665.1439, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 9.7347, p-val = 0.0018
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.3444 0.0592 -5.8215 <.0001 -0.4603 -0.2284 ***
## managementyes 0.3485 0.1117 3.1201 0.0018 0.1296 0.5674 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
subj_vs_obj_model
##
## Multivariate Meta-Analysis Model (k = 187; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0458 0.2141 92 no study_id
## sigma^2.2 0.2852 0.5340 187 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 185) = 19218.5561, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 4.1856, p-val = 0.0408
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2899 0.0538 -5.3913 <.0001 -0.3953 -0.1845 ***
## subj_vs_objsubjective 0.2382 0.1164 2.0459 0.0408 0.0100 0.4664 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
severity_model
##
## Multivariate Meta-Analysis Model (k = 176; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0737 0.2715 89 no study_id
## sigma^2.2 0.2577 0.5076 176 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 173) = 19100.0392, p-val < .0001
##
## Test of Moderators (coefficients 2:3):
## QM(df = 2) = 20.3878, p-val < .0001
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt 0.0947 0.1031 0.9187 0.3583 -0.1074 0.2968
## severitymedium -0.3376 0.1269 -2.6613 0.0078 -0.5863 -0.0890 **
## severityhigh -0.5959 0.1320 -4.5151 <.0001 -0.8545 -0.3372 ***
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
human_type_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0445 0.2110 99 no study_id
## sigma^2.2 0.2410 0.4910 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 204) = 20015.5297, p-val < .0001
##
## Test of Moderators (coefficients 2:7):
## QM(df = 6) = 26.0937, p-val = 0.0002
##
## Model Results:
##
## estimate se zval pval ci.lb
## intrcpt -0.2207 0.1456 -1.5162 0.1295 -0.5061
## human_typeexpert(s) -0.1273 0.1534 -0.8295 0.4068 -0.4280
## human_typehuman-controlled machine -0.0376 0.3144 -0.1197 0.9047 -0.6538
## human_typemultiple 0.2787 0.2973 0.9374 0.3485 -0.3040
## human_typeother 0.0703 0.4379 0.1605 0.8725 -0.7880
## human_typeother participant(s) 0.3605 0.2028 1.7779 0.0754 -0.0369
## human_typeparticipants themselves 0.9534 0.3077 3.0982 0.0019 0.3503
## ci.ub
## intrcpt 0.0646
## human_typeexpert(s) 0.1735
## human_typehuman-controlled machine 0.5785
## human_typemultiple 0.8613
## human_typeother 0.9286
## human_typeother participant(s) 0.7580 .
## human_typeparticipants themselves 1.5566 **
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
expert_model
##
## Multivariate Meta-Analysis Model (k = 202; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0618 0.2486 92 no study_id
## sigma^2.2 0.2357 0.4854 202 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 200) = 20003.5319, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 5.3091, p-val = 0.0212
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.0962 0.0939 -1.0246 0.3055 -0.2801 0.0878
## expertyes -0.2476 0.1075 -2.3041 0.0212 -0.4583 -0.0370 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
algorithm_dummy_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0735 0.2711 99 no study_id
## sigma^2.2 0.2470 0.4970 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20615.2367, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 5.4307, p-val = 0.0198
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.3511 0.0653 -5.3789 <.0001 -0.4791 -0.2232 ***
## algorithm_dummyyes 0.2200 0.0944 2.3304 0.0198 0.0350 0.4051 *
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
ai_dummy_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0699 0.2643 99 no study_id
## sigma^2.2 0.2524 0.5024 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20901.3765, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 3.7717, p-val = 0.0521
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.2992 0.0544 -5.5024 <.0001 -0.4058 -0.1926 ***
## ai_dummyyes 0.2134 0.1099 1.9421 0.0521 -0.0020 0.4288 .
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
textual_model
##
## Multivariate Meta-Analysis Model (k = 211; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0783 0.2799 99 no study_id
## sigma^2.2 0.2511 0.5011 211 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 209) = 20499.3534, p-val < .0001
##
## Test of Moderators (coefficient 2):
## QM(df = 1) = 0.7947, p-val = 0.3727
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -0.1503 0.1180 -1.2735 0.2028 -0.3817 0.0810
## textualyes -0.1153 0.1293 -0.8914 0.3727 -0.3688 0.1382
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
multilevelmodel
##
## Multivariate Meta-Analysis Model (k = 179; method: ML)
##
## Variance Components:
##
## estim sqrt nlvls fixed factor
## sigma^2.1 0.0311 0.1764 83 no study_id
## sigma^2.2 0.2115 0.4599 179 no study_id/es_id
##
## Test for Residual Heterogeneity:
## QE(df = 166) = 14236.0668, p-val < .0001
##
## Test of Moderators (coefficients 2:13):
## QM(df = 12) = 63.0851, p-val < .0001
##
## Model Results:
##
## estimate se zval pval ci.lb ci.ub
## intrcpt -93.0234 36.0786 -2.5784 0.0099 -163.7362 -22.3105
## pubyear 0.0454 0.0179 2.5399 0.0111 0.0104 0.0804
## within_subject 0.0876 0.0786 1.1152 0.2647 -0.0664 0.2417
## dv_measurement_type 0.1599 0.0712 2.2461 0.0247 0.0204 0.2995
## preregistered -0.0011 0.0602 -0.0190 0.9849 -0.1191 0.1168
## m_age 0.0234 0.0075 3.1226 0.0018 0.0087 0.0381
## percntg_females 0.0151 0.0042 3.6240 0.0003 0.0069 0.0233
## online -0.1571 0.0588 -2.6715 0.0076 -0.2723 -0.0418
## US 0.0728 0.0631 1.1536 0.2487 -0.0509 0.1966
## management 0.2627 0.0555 4.7339 <.0001 0.1539 0.3714
## expert -0.2215 0.0662 -3.3474 0.0008 -0.3512 -0.0918
## ai_dummy 0.0433 0.0600 0.7209 0.4710 -0.0744 0.1609
## textual 0.0053 0.0676 0.0788 0.9372 -0.1273 0.1379
##
## intrcpt **
## pubyear *
## within_subject
## dv_measurement_type *
## preregistered
## m_age **
## percntg_females ***
## online **
## US
## management ***
## expert ***
## ai_dummy
## textual
##
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
funnel(multilevelmodel, yaxis = "sei", main = "Funnel Plot Multi-level Random-Effects Model Including Moderators", label = 5, offset=1, back = "lightgrey", pch = 20)
cordata <- select(meta_data, pubyear, within_subject, dv_measurement_type, preregistered, m_age, percntg_females, online, US, management, expert, ai_dummy, textual)
head(cordata)
## pubyear within_subject dv_measurement_type preregistered m_age
## 1 2019 -1 -1 -1 36.0
## 2 2019 -1 1 -1 NA
## 3 2019 -1 1 -1 NA
## 4 2019 -1 -1 -1 19.6
## 5 2019 -1 -1 -1 33.6
## 6 2019 -1 -1 -1 36.9
## percntg_females online US management expert ai_dummy textual
## 1 45.0 1 0 0 1 -1 1
## 2 40.0 -1 0 -1 1 -1 -1
## 3 40.0 -1 0 -1 1 -1 -1
## 4 54.9 -1 1 -1 1 -1 1
## 5 55.3 1 0 -1 1 1 1
## 6 48.1 1 0 -1 1 -1 1
res <- cor(cordata, use = "complete.obs")
round(res, 2)
## pubyear within_subject dv_measurement_type preregistered
## pubyear 1.00 -0.42 0.18 0.29
## within_subject -0.42 1.00 0.22 0.14
## dv_measurement_type 0.18 0.22 1.00 0.30
## preregistered 0.29 0.14 0.30 1.00
## m_age 0.19 -0.23 -0.14 0.16
## percntg_females -0.27 0.09 -0.34 -0.18
## online 0.50 -0.15 0.03 0.36
## US 0.35 -0.09 0.11 0.23
## management -0.08 -0.24 -0.05 -0.27
## expert -0.09 -0.04 -0.31 -0.14
## ai_dummy 0.35 -0.08 0.13 -0.06
## textual -0.16 -0.30 -0.47 -0.36
## m_age percntg_females online US management expert
## pubyear 0.19 -0.27 0.50 0.35 -0.08 -0.09
## within_subject -0.23 0.09 -0.15 -0.09 -0.24 -0.04
## dv_measurement_type -0.14 -0.34 0.03 0.11 -0.05 -0.31
## preregistered 0.16 -0.18 0.36 0.23 -0.27 -0.14
## m_age 1.00 -0.25 0.28 0.05 0.12 0.27
## percntg_females -0.25 1.00 -0.23 -0.19 -0.29 0.04
## online 0.28 -0.23 1.00 0.42 0.04 -0.07
## US 0.05 -0.19 0.42 1.00 -0.02 -0.15
## management 0.12 -0.29 0.04 -0.02 1.00 0.35
## expert 0.27 0.04 -0.07 -0.15 0.35 1.00
## ai_dummy 0.16 -0.27 -0.04 0.03 -0.16 -0.32
## textual -0.02 0.32 -0.12 -0.15 0.18 0.26
## ai_dummy textual
## pubyear 0.35 -0.16
## within_subject -0.08 -0.30
## dv_measurement_type 0.13 -0.47
## preregistered -0.06 -0.36
## m_age 0.16 -0.02
## percntg_females -0.27 0.32
## online -0.04 -0.12
## US 0.03 -0.15
## management -0.16 0.18
## expert -0.32 0.26
## ai_dummy 1.00 -0.08
## textual -0.08 1.00
library(corrplot)
## corrplot 0.92 loaded
corrplot(res, type = "upper", order = "hclust",
tl.col = "black", tl.srt = 45)
library("PerformanceAnalytics")
## Loading required package: xts
## Loading required package: zoo
##
## Attaching package: 'zoo'
## The following objects are masked from 'package:base':
##
## as.Date, as.Date.numeric
##
## Attaching package: 'xts'
## The following objects are masked from 'package:dplyr':
##
## first, last
##
## Attaching package: 'PerformanceAnalytics'
## The following object is masked from 'package:graphics':
##
## legend
chart.Correlation(cordata, histogram=TRUE, pch=19)
# Standardized residuals (observed residuals divided by the corresponding standard errors)
standresid <- residuals(multilevel, type="rstandard")
# Cook's distance D for each observed effect size (change in beta if one observation is dropped)
# Thresholds: D > 0.5 and > 1.0; D > 3*mean; D > 4/n; D > chisq(# regrcoeff, 0.50); any extreme D-value
cookdist <- cooks.distance(multilevel, progbar = TRUE, reestimate=FALSE)